import requests
import scrapy
from spidertools.utils.time_utils import get_current_date
import json
from commonresources.inner_utils.standardize_field_utils import check_city_field
from commonresources.spider_items.henan.items import HeNanShengDianZiZhaoBiaoTouBiaoGongGongFuWuPingTaiItem
from commonresources.spiders.basespider import BaseSpider
import time


class HeNanShengDianZiZhaoBiaoTouBiaoGongGongFuWuPingTaiSpider(BaseSpider):
    name = 'HeNanShengDianZiZhaoBiaoTouBiaoGongGongFuWuPingTai'
    name_zh = "河南省电子招标投标公共服务平台"
    province = "河南"

    start_urls = ['http://hndzzbtb.hndrc.gov.cn/002/tradePublic.html#']

    def __init__(self, full_dose=False):
        super(HeNanShengDianZiZhaoBiaoTouBiaoGongGongFuWuPingTaiSpider, self).__init__(full_dose)


    def start_requests(self):
        type_list = ['002001001', '002001002', '002001003', '002001004']
        # type_list = ['002001004']
        for type in type_list:
            start_url = 'http://hndzzbtb.hndrc.gov.cn/services/hl/getSelect?response=application/json&pageIndex=1&pageSize=22&day=&sheng=x1&qu=&xian=&title=&timestart=&timeend=&categorynum=' + type + '&siteguid=3955b792-fb32-4dc1-8935-49ad516ae6db'
            if type=='002001001':
                page=5000
            elif type=='002001002':#设置页数的预定值
                page=10
            elif type=='002001003':
                page=3000
            elif type=='002001004':
                page=4000
            yield scrapy.Request(
                url=start_url,
                method="get",
                meta={
                    "need_break": False,
                    'type':type,
                    'page':page
                },
                callback=self.first_parse,
            )

    def first_parse(self,response):
        second_page_number = 1
        headers = {
            'USER_AGENT': "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"
        }
        if not response.meta['need_break']:
            while second_page_number < int(response.meta['page']):
                start_url = 'http://hndzzbtb.hndrc.gov.cn/services/hl/getSelect?response=application/json&pageIndex=%d' % (
                    second_page_number) + '&pageSize=22&day=&sheng=x1&qu=&xian=&title=&timestart=&timeend=&categorynum=' + response.meta['type'] + '&siteguid=3955b792-fb32-4dc1-8935-49ad516ae6db'
                print(second_page_number)
                if not response.meta['need_break']:
                    html = requests.get(url=start_url, headers=headers).text
                    html = json.loads(html)
                    html = html['return']
                    html = json.loads(html)['Table']
                    for i in html:
                        url = 'http://hndzzbtb.hndrc.gov.cn' + i['href']
                        type = i['category']
                        if type == '002001001':
                            type = '招标公告'
                        elif type == '002001002':
                            type = '变更公告'
                        elif type == '002001003':
                            type = '评标结果公告'
                        elif type == '002001004':
                            type = '中标公告'
                        title = i['title']
                        infodate = i['infodate']
                        city = i['infoc']
                        fin_html = requests.get(url=url).text
                        item = HeNanShengDianZiZhaoBiaoTouBiaoGongGongFuWuPingTaiItem()
                        item['origin_url'] = url#原始网站
                        item['announcement_title'] = title#公告标题
                        item['city'] = city#城市
                        item['release_time'] = infodate#发布时间
                        item['announcement_type'] = type#公告类型
                        item['html'] = fin_html#网页源码
                        item['source_type']='河南省电子招标投标公共服务平台'
                        item['province']='河南'
                        item['is_parsed'] = 0
                        if not self.full_dose and item['release_time'] != get_current_date():
                            # if not self.full_dose and item['release_time'] != "2021-02-22":
                            response.meta['need_break'] = True
                        else:
                            yield item
            # time.sleep(1)
                if not response.meta['need_break']:
                    second_page_number += 1
                else:
                    break




